import time
from requests.exceptions import ConnectTimeout #自定义异常
from requests.exceptions import RequestException

import requests
import json
import  os
from lxml import etree
from requests.exceptions import ConnectionError
from requests.exceptions import ChunkedEncodingError

#url1 = "https://www.denverartmuseum.org/en/object/1992.559"
url2 = "http://museum.asiasociety.org/collection/explore/search/search:china--from-date:-3000--to-date:1950/page/"

#print(requests.get(url1))
#print(requests.get(url2))
headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.75 Safari/537.36 Edg/100.0.1185.36' }
param={
    'Accept': 'image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8',
    'Accept-Encoding': 'gzip, deflate',
    'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
    'Host': 'ping.chartbeat.net',
    'Proxy-Connection': 'keep-alive',
    'Referer': 'http://museum.asiasociety.org/'
}
i=1
idj=0
for i in range(1,7):
    newurl2=url2+str(i)
    while True:  # 一直循环，知道访问站点成功
        try:
            pagetext = requests.get(url=newurl2, params=param, headers=headers).text
            break
        except ConnectionError as e:  # e=Exception() 定义一个异常对象
            print("ConnectionError")
            time.sleep(3)
        except ChunkedEncodingError as e:
            print("ChunkedEncodingError")
            time.sleep(3)
        except Exception as e:
            print("Exception")
            time.sleep(3)
            time.sleep(3)
    print("page:"+str(i))
    tree = etree.HTML(pagetext)
    list = tree.xpath("//div[@class='highlights isotope-grid']/div")
    j = 0
    for li in list:

        j = j + 1
        src=li.xpath("../div["+str(j)+"]/h3/a/@href")
        src1=src[0]

        #print(j)
        while True:  # 一直循环，知道访问站点成功
            try:
                ptext = requests.get(url=src1, headers=headers).text
                #print(ptext)
                break
            except ConnectionError as e:  # e=Exception() 定义一个异常对象
                print("ConnectionError")
                time.sleep(3)
            except ChunkedEncodingError as e:
                print("ChunkedEncodingError")
                time.sleep(3)
            except Exception as e:
                print("Exception")
                time.sleep(3)
                time.sleep(3)
        tree1 = etree.HTML(ptext)
        patext=tree1.xpath("//div[@class='section-block object-intro nb']//text()")
        print(str(patext))
        idj=idj+1
        print(idj)
        filepath="./json2/json2/"+str(idj)+".text"
        with open(filepath, 'w', encoding='utf-8') as fp:
            fp.write(str(patext))







print("#####")

